# conv-s2s
CUDA_VISIBLE_DEVICES=1 python eval_lm_item.py --task translation_bpe data-bin/writingPrompts-prompt2story_bpe500_fix \
    --path ../fire_data/running/conv-s2s-d0.3_bpe500/checkpoint_best.pt \
    --user-dir coherence_story --max-tokens 4000 \
    --skip-invalid-size-inputs-valid-test \
    --results-path ../fire_data/running/conv-s2s-d0.3_bpe500 \
    --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_src.txt \
    --external-tgt-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_tgt.txt
CUDA_VISIBLE_DEVICES=3 python eval_lm_item.py --task translation_bpe data-bin/writingPrompts-prompt2story_bpe500_fix \
    --path ../fire_data/running/conv-s2s-d0.3_bpe500/checkpoint_best.pt \
    --user-dir coherence_story --max-tokens 4000 \
    --skip-invalid-size-inputs-valid-test \
    --results-path ../fire_data/running/conv-s2s-d0.3_bpe500 \
    --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/coherence_src.txt \
    --external-tgt-path /home/rickwwang/project_own/story_generation/data/evaluate/coherence_tgt.txt

# conv-s2s fusion
CUDA_VISIBLE_DEVICES=0 python eval_lm_item.py --task translation_bpe data-bin/writingPrompts-prompt2story_bpe500_fix \
    --path ../fire_data/running/conv-s2s-d0.3-fusion_bpe500/checkpoint_best.pt \
    --user-dir coherence_story --max-tokens 4000 \
    --skip-invalid-size-inputs-valid-test \
    --results-path ../fire_data/running/conv-s2s-d0.3-fusion_bpe500 \
    --model-overrides "{'pretrained_checkpoint':'../fire_data/running/conv-s2s-d0.3_bpe500/checkpoint_best.pt'}" \
    --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_src.txt \
    --external-tgt-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_tgt.txt
CUDA_VISIBLE_DEVICES=0 python eval_lm_item.py --task translation_bpe data-bin/writingPrompts-prompt2story_bpe500_fix \
    --path ../fire_data/running/conv-s2s-d0.3-fusion_bpe500/checkpoint_best.pt \
    --user-dir coherence_story --max-tokens 4000 \
    --skip-invalid-size-inputs-valid-test \
    --results-path ../fire_data/running/conv-s2s-d0.3-fusion_bpe500 \
    --model-overrides "{'pretrained_checkpoint':'../fire_data/running/conv-s2s-d0.3_bpe500/checkpoint_best.pt'}" \
    --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/coherence_src.txt \
    --external-tgt-path /home/rickwwang/project_own/story_generation/data/evaluate/coherence_tgt.txt

# story language model
CUDA_VISIBLE_DEVICES=2 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format2/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format2 \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_src_tgt.txt \
  --use-context True
CUDA_VISIBLE_DEVICES=3 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format2/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format2 \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/coherence_tgt.txt

CUDA_VISIBLE_DEVICES=2 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2 \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_src_tgt.txt \
  --use-context True
CUDA_VISIBLE_DEVICES=2 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2 \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/coherence_tgt.txt

# other prompt rank and coherence rank
CUDA_VISIBLE_DEVICES=2 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gpt/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gpt \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_src_tgt.txt \
  --use-context True
CUDA_VISIBLE_DEVICES=0 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gpt/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gpt \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/coherence_tgt.txt

CUDA_VISIBLE_DEVICES=2 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_gpt/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_gpt \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_src_tgt.txt \
  --use-context True
CUDA_VISIBLE_DEVICES=1 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_gpt/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_gpt \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/coherence_tgt.txt

CUDA_VISIBLE_DEVICES=6 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_coherence_gpt/checkpoint30.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_coherence_gpt \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_src_tgt.txt \
  --use-context True
CUDA_VISIBLE_DEVICES=2 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_coherence_gpt/checkpoint30.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_coherence_gpt \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/coherence_tgt.txt

CUDA_VISIBLE_DEVICES=7 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_coherence_gpt/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_coherence_gpt \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_src_tgt.txt \
  --use-context True
CUDA_VISIBLE_DEVICES=3 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_coherence_gpt/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../fire_data/running/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_coherence_gpt \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/coherence_tgt.txt

CUDA_VISIBLE_DEVICES=2 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../out/transformer-lm_bpe500_warmlr_format_pre2_memory/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../out/transformer-lm_bpe500_warmlr_format_pre2_memory \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_src_tgt.txt \
  --use-context True
CUDA_VISIBLE_DEVICES=4 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../out/transformer-lm_bpe500_warmlr_format_pre2_memory/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../out/transformer-lm_bpe500_warmlr_format_pre2_memory \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/coherence_tgt.txt

CUDA_VISIBLE_DEVICES=4 python eval_lm_item.py --task language_modeling_bpe data-bin/writingPrompts-event2story_bpe500 \
  --path ../out/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_relevance_gpt_fix/checkpoint_best.pt \
  --user-dir coherence_story --max-tokens 4000 \
  --skip-invalid-size-inputs-valid-test --tokens-per-sample 1024 --sample-break-mode eos \
  --results-path ../out/transformer-lm_bpe500_warmlr_format_pre2_memory_gate_relevance_gpt_fix \
  --external-data --external-src-path /home/rickwwang/project_own/story_generation/data/evaluate/prompt_rank_src_tgt.txt \
  --use-context True